{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "adf7aa20",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-29T08:22:17.998747Z",
     "start_time": "2023-10-29T08:22:16.797633Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5z/nh50kgqx74d2dzsth3mspb9w0000gn/T/ipykernel_35803/3998943726.py:4: FutureWarning: list_metrics is deprecated and will be removed in the next major version of datasets. Use 'evaluate.list_evaluation_modules' instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluate\n",
      "  metrics_list = list_metrics()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(137,\n",
       " ['accuracy',\n",
       "  'bertscore',\n",
       "  'bleu',\n",
       "  'bleurt',\n",
       "  'brier_score',\n",
       "  'cer',\n",
       "  'character',\n",
       "  'charcut_mt',\n",
       "  'chrf',\n",
       "  'code_eval',\n",
       "  'comet',\n",
       "  'competition_math',\n",
       "  'coval',\n",
       "  'cuad',\n",
       "  'exact_match',\n",
       "  'f1',\n",
       "  'frugalscore',\n",
       "  'glue',\n",
       "  'google_bleu',\n",
       "  'indic_glue',\n",
       "  'mae',\n",
       "  'mahalanobis',\n",
       "  'mape',\n",
       "  'mase',\n",
       "  'matthews_correlation',\n",
       "  'mauve',\n",
       "  'mean_iou',\n",
       "  'meteor',\n",
       "  'mse',\n",
       "  'nist_mt',\n",
       "  'pearsonr',\n",
       "  'perplexity',\n",
       "  'poseval',\n",
       "  'precision',\n",
       "  'r_squared',\n",
       "  'recall',\n",
       "  'rl_reliability',\n",
       "  'roc_auc',\n",
       "  'rouge',\n",
       "  'sacrebleu',\n",
       "  'sari',\n",
       "  'seqeval',\n",
       "  'smape',\n",
       "  'spearmanr',\n",
       "  'squad',\n",
       "  'squad_v2',\n",
       "  'super_glue',\n",
       "  'ter',\n",
       "  'trec_eval',\n",
       "  'wer',\n",
       "  'wiki_split',\n",
       "  'xnli',\n",
       "  'xtreme_s',\n",
       "  'AlhitawiMohammed22/CER_Hu-Evaluation-Metrics',\n",
       "  'BucketHeadP65/confusion_matrix',\n",
       "  'BucketHeadP65/roc_curve',\n",
       "  'DarrenChensformer/eval_keyphrase',\n",
       "  'DarrenChensformer/relation_extraction',\n",
       "  'Drunper/metrica_tesi',\n",
       "  'Felipehonorato/eer',\n",
       "  'He-Xingwei/sari_metric',\n",
       "  'Ikala-allen/relation_extraction',\n",
       "  'JP-SystemsX/nDCG',\n",
       "  'Josh98/nl2bash_m',\n",
       "  'Muennighoff/code_eval_octopack',\n",
       "  'NCSOFT/harim_plus',\n",
       "  'Natooz/ece',\n",
       "  'NikitaMartynov/spell-check-metric',\n",
       "  'Pipatpong/perplexity',\n",
       "  'SpfIo/wer_checker',\n",
       "  'Splend1dchan/cosine_similarity',\n",
       "  'Vallp/ter',\n",
       "  'Viona/fuzzy_reordering',\n",
       "  'Viona/kendall_tau',\n",
       "  'Vipitis/shadermatch',\n",
       "  'Yeshwant123/mcc',\n",
       "  'abdusah/aradiawer',\n",
       "  'abidlabs/mean_iou',\n",
       "  'abidlabs/mean_iou2',\n",
       "  'amitness/perplexity',\n",
       "  'andstor/code_perplexity',\n",
       "  'angelina-wang/directional_bias_amplification',\n",
       "  'aryopg/roc_auc_skip_uniform_labels',\n",
       "  'bdsaglam/jer',\n",
       "  'brian920128/doc_retrieve_metrics',\n",
       "  'bstrai/classification_report',\n",
       "  'bugbounty1806/accuracy',\n",
       "  'carletoncognitivescience/peak_signal_to_noise_ratio',\n",
       "  'chanelcolgate/average_precision',\n",
       "  'ckb/unigram',\n",
       "  'codeparrot/apps_metric',\n",
       "  'cpllab/syntaxgym',\n",
       "  'danieldux/hierarchical_softmax_loss',\n",
       "  'dvitel/codebleu',\n",
       "  'ecody726/bertscore',\n",
       "  'fschlatt/ner_eval',\n",
       "  'gabeorlanski/bc_eval',\n",
       "  'giulio98/codebleu',\n",
       "  'guydav/restrictedpython_code_eval',\n",
       "  'harshhpareek/bertscore',\n",
       "  'hpi-dhc/FairEval',\n",
       "  'hynky/sklearn_proxy',\n",
       "  'hyperml/balanced_accuracy',\n",
       "  'illorca/FairEval',\n",
       "  'ingyu/klue_mrc',\n",
       "  'jjkim0807/code_eval',\n",
       "  'jpxkqx/peak_signal_to_noise_ratio',\n",
       "  'jpxkqx/signal_to_reconstruction_error',\n",
       "  'k4black/codebleu',\n",
       "  'kaggle/ai4code',\n",
       "  'kedudzic/charmatch',\n",
       "  'langdonholmes/cohen_weighted_kappa',\n",
       "  'lhy/hamming_loss',\n",
       "  'lhy/ranking_loss',\n",
       "  'lvwerra/accuracy_score',\n",
       "  'manueldeprada/beer',\n",
       "  'mfumanelli/geometric_mean',\n",
       "  'mtc/fragments',\n",
       "  'nlpln/tst',\n",
       "  'omidf/squad_precision_recall',\n",
       "  'posicube/mean_reciprocal_rank',\n",
       "  'repllabs/mean_average_precision',\n",
       "  'repllabs/mean_reciprocal_rank',\n",
       "  'sakusakumura/bertscore',\n",
       "  'shalakasatheesh/squad',\n",
       "  'sma2023/wil',\n",
       "  'tialaeMceryu/unigram',\n",
       "  'transZ/sbert_cosine',\n",
       "  'transZ/test_parascore',\n",
       "  'transformersegmentation/segmentation_scores',\n",
       "  'unitxt/metric',\n",
       "  'unnati/kendall_tau_distance',\n",
       "  'vichyt/metric-codebleu',\n",
       "  'weiqis/pajm',\n",
       "  'ybelkada/cocoevaluate',\n",
       "  'yonting/average_precision_score',\n",
       "  'yuyijiong/quad_match_score'])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import list_metrics\n",
    "\n",
    "#列出评价指标\n",
    "metrics_list = list_metrics()\n",
    "\n",
    "len(metrics_list), metrics_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fb6bafa9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-29T08:22:22.976604Z",
     "start_time": "2023-10-29T08:22:18.002658Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Computes BLEU score of translated segments against one or more references.\n",
      "Args:\n",
      "    predictions: list of translations to score.\n",
      "    references: list of lists of or just a list of references for each translation.\n",
      "    tokenizer : approach used for tokenizing `predictions` and `references`.\n",
      "        The default tokenizer is `tokenizer_13a`, a minimal tokenization approach that is equivalent to `mteval-v13a`, used by WMT.\n",
      "        This can be replaced by any function that takes a string as input and returns a list of tokens as output.\n",
      "    max_order: Maximum n-gram order to use when computing BLEU score.\n",
      "    smooth: Whether or not to apply Lin et al. 2004 smoothing.\n",
      "Returns:\n",
      "    'bleu': bleu score,\n",
      "    'precisions': geometric mean of n-gram precisions,\n",
      "    'brevity_penalty': brevity penalty,\n",
      "    'length_ratio': ratio of lengths,\n",
      "    'translation_length': translation_length,\n",
      "    'reference_length': reference_length\n",
      "Examples:\n",
      "\n",
      "    >>> predictions = [\"hello there general kenobi\", \"foo bar foobar\"]\n",
      "    >>> references = [\n",
      "    ...     [\"hello there general kenobi\", \"hello there!\"],\n",
      "    ...     [\"foo bar foobar\"]\n",
      "    ... ]\n",
      "    >>> bleu = evaluate.load(\"bleu\")\n",
      "    >>> results = bleu.compute(predictions=predictions, references=references)\n",
      "    >>> print(results[\"bleu\"])\n",
      "    1.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import evaluate\n",
    "\n",
    "blue_evaluate = evaluate.load('bleu')\n",
    "print(blue_evaluate.inputs_description)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8538d6a0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-10-29T08:22:23.071889Z",
     "start_time": "2023-10-29T08:22:23.064706Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bleu': 0.0,\n",
       " 'precisions': [1.0, 0.0, 0.0, 0.0],\n",
       " 'brevity_penalty': 0.36787944117144233,\n",
       " 'length_ratio': 0.5,\n",
       " 'translation_length': 2,\n",
       " 'reference_length': 4}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#计算一个评价指标\n",
    "predictions = ['Hello', 'World']\n",
    "references = ['Hello World!', 'World']\n",
    "\n",
    "score = blue_evaluate.compute(predictions=predictions, references=references)\n",
    "score"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python(d2l)",
   "language": "python",
   "name": "d2l"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
